 /** 
* Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package com.tompai.datagather;

import java.io.IOException;

import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.tompai.datagather.common.MybatisUtil;
import com.tompai.datagather.dao.CankaoxiaoxiMapper;
import com.tompai.datagather.model.Cankaoxiaoxi;

/**
* @desc: datagather-tools
* @name: CankaoCrawlerDemo.java
* @author: tompai
* @createTime: 2018年5月16日 下午3:22:11
* @history:
* @version: v1.0
*/

public class CankaoCrawlerDemo {

	private static final Logger logger = Logger.getLogger(CankaoCrawlerDemo.class);
	/**
	* @author: tompai
	* @createTime: 2018年5月16日 下午3:22:11
	* @history:
	* @param args void
	*/

	public static void main(String[] args) {
		//TODO Auto-generated method stub

		String cankaoUrl = "http://www.cankaoxiaoxi.com/mil/zgjq/";
		Document doc;
		try {
			doc = Jsoup.connect(cankaoUrl)
					.userAgent("Mozilla")
					.cookie("auth", "token")
					.timeout(3000)
					.get();

			//String title = doc.title();
			//logger.info("Title:" + title);

			Elements links = doc.select(".txt-list-a.fz-14>li");//
			//数据库操作//
			CankaoxiaoxiMapper canMapper;
			canMapper = MybatisUtil.getSession().getMapper(CankaoxiaoxiMapper.class);
			Cankaoxiaoxi cankaos = new Cankaoxiaoxi();
			
			for (Element element : links) {
				
				String time=element.select(".f-r.arial.cor999").text();
				
				String canktile=element.select("a").text();
				String link=element.select("a").attr("href");
				
				//database//
				if(StringUtils.isNoneEmpty(canktile)) {
					cankaos.setTitle(canktile);
				}
				if(StringUtils.isNoneEmpty(link)) {
					cankaos.setUrl(link);
				}
				if(cankaos!=null) {
					
					//Cankaoxiaoxi cans= canMapper.selectByTile(canktile);
					//cans.getTitle();;
					canMapper.insert(cankaos);
					MybatisUtil.getSession().commit();//
				}
				logger.info("time:" + time);
				logger.info("canktile:" + canktile);
				logger.info("link:" + link);
			}
			
		} catch (IOException e) {
			// TODO: handle exception
			logger.error(e.getMessage());
		}
	}

}

